#!/usr/bin/perl
use warnings;
use strict;
BEGIN {
    unshift @INC, '../modules';
}
use File::Path;
use File::Spec::Functions;
use Storable;
use TFIDF;
use Readonly;
Readonly my $rocchio_alpha => 16;
Readonly my $rocchio_beta  => 4;

our %params = (
    dmoz_dir          => '/home/xuqy/dmoz',
    sample_tfidf_file => 'data/tfidf',
    idf_file      => 'data/idf',
    svm_learn         => 'tools/svm_learn',
    svm_classify      => 'tools/svm_classify',
    lpu_file          => 'tools/lpu',
    download_script   => 'scripts/batch_download.pl',

    topics_dir        => 'topics2',
    list_file         => 'list',
    threads_cnt       => 50,
    classifier_file   => 'classifier',
    seeds_file        => 'seeds',
    dump_dir          => 'dump',
    positive_tfidf_file  => 'pos_tfidf',
    svm_model_file    => 'svm_model',
    min_page_len      => 1_000,
    max_page_len      => 128_000,
);

unless ( @ARGV == 1 ) {
    print STDERR "Usage: $0 <topic>\n";
    die;
}

my $dmoz_dir = $params{dmoz_dir};
chdir $dmoz_dir or die "$!";

$params{$_} = File::Spec->rel2abs( $params{$_} )
  for qw(download_script sample_tfidf_file idf_file svm_learn svm_classify lpu_file);

my $topic = shift;

die "Your dir doesn't exists!\n"
  unless -d $topic;

( my $topic_dir = $topic ) =~ s/\//\./g;
my $topics_dir = $params{topics_dir};
mkdir $topics_dir unless -d $topics_dir;
$topic_dir = "$topics_dir/$topic_dir";
mkdir $topic_dir unless -d $topic_dir;
print STDERR "Collecting seed URLs...\n";
my %seed_urls = %{ &collect_seeds($topic) };

chdir $topic_dir or die "$!";

open SEEDS, "> $params{seeds_file}" or die "$!";
print SEEDS $_, "\n" for keys %seed_urls;
close SEEDS;
printf STDERR "Done! %d seeds URLs collected.\n", scalar( keys %seed_urls );

unless (-d $params{dump_dir}) {
    mkdir $params{dump_dir};
    print STDERR "Start downloading seed URLs...\n";
    system $params{download_script}, $params{seeds_file}, $params{dump_dir}, $params{threads_cnt};
    print STDERR "Done!\n";
}

open PROBLEM, "> $params{positive_tfidf_file}"       or die "$!";

my $idf_ref = Storable::retrieve($params{idf_file});
my $total = $idf_ref->{TOTAL};

my $dump_dir = $params{dump_dir};
opendir DIR, $dump_dir or die "$!";
foreach my $entry ( readdir DIR ) {
    next if $entry =~ /^\.{1,2}$/;
    my $file = "$dump_dir/$entry";
    next if -s $file < $params{min_page_len} or -s $file > $params{max_page_len};
    open my $in, '<', $file or die "$!";
    my $html = do { local $/; <$in> };
    my $tfidf_ref = TFIDF::html2tfidf($html, $idf_ref, $total);
    if ( @$tfidf_ref ) {
        print PROBLEM join(' ', map {sprintf("%d:%.6f", @$_)} @$tfidf_ref), "\n";
    }
    close $in;
}
closedir DIR;
close PROBLEM;

print STDERR "start LPU learning...\n";

system $params{lpu_file}, $params{positive_tfidf_file}, $params{sample_tfidf_file}, $params{svm_learn}, $params{svm_classify}, $params{svm_model_file};

print STDERR "Done!";

my @index2pair;
while ( my ( $token, $ref ) = each %$idf_ref ) {
    next if $token eq 'TOTAL';
    my ( $index, $idf ) = @$ref;
    $index2pair[$index] = [ $token, $idf ];
}
undef $idf_ref;

my $bias;
open IN, "< $params{svm_model_file}" or die "$!";
while (<IN>) {
  if (/^(\S+) # threshold/) {
    $bias = $1;
    last;
  }
}

my @weights;

while (<IN>) {
  /^(\S+) (.+) #/;
  my $alpha = $1;
  my $pairs = $2;
  next unless defined $pairs;
  while ($pairs =~ /(\S+):(\S+)/g) {
    $weights[$1] += $alpha * $2;
  }
}
close IN;

my %classifier;
for (1..$#weights) {
  next unless $weights[$_];
  my ($token, $idf) = @{$index2pair[$_]};
  my $factor = sprintf("%.6f", log($total/$idf));
  $classifier{$token} = [$weights[$_], $factor];
}
$classifier{BIAS} = $bias;

Storable::store(\%classifier, $params{classifier_file});
unlink @params{'positive_tfidf_file', 'svm_model_file'};
system 'rm -rf dump';

print STDERR "Done!\n";

sub collect_seeds {
    my ($topic_root) = shift;
    my $seeds_ref = {};
    my @dirs = ( $topic_root );
    while (@dirs) {
        my $dir = shift @dirs;
        print STDERR $dir, "\n";
        next unless defined $dir;
        my $list_file = catfile( $dir, $params{list_file} );
        if ( -e $list_file ) {
            open IN, "< $list_file" or die "$!";
            while (<IN>) {
                chomp;
                $seeds_ref->{$_} = 1;
            }
            close IN;
        }
        opendir DIR, $dir or die "$!";
        my @sub_dirs;
        foreach my $entry ( readdir DIR ) {
            next if $entry =~ /^\.{1,2}$/;
            my $sub_dir = catfile( $dir, $entry );
            push @sub_dirs, $sub_dir if -d $sub_dir and not -l $sub_dir;
        }
        closedir DIR;
        unshift @dirs, @sub_dirs;
    }
    return $seeds_ref;
}

sub collect_weights {
    my ($file_name, $weights_ref) = @_;
    my $cnt = 0;
    open IN, "< $file_name" or die "$!";
    while (<IN>) {
        $cnt++;
        while (/(\S+):(\S+)/g) {
            $weights_ref->[$1] += $2
        }
    }
    close IN;
    return $cnt;
}

sub normalize {
    my $vector_ref = shift;
    my $norm2 = 0;
    for (@$vector_ref) {
        $norm2 += $_->[1] ** 2;
    }
    $norm2 = sqrt $norm2;
    for (@$vector_ref) {
        $_->[1] = sprintf("%.6f", $_->[1] / $norm2);
    }
}

sub get_similarity {
    my ($vec1, $vec2) = @_;
    return 0 unless @$vec1 and @$vec2;
    my ($index1, $index2) = (0, 0);
    my $sim = 0;
    while ($index1 < $#{$vec1} and $index2 < $#{$vec2}) {
        if ($vec1->[$index1][0] == $vec2->[$index2][0]) {
            $sim += sprintf("%.6f", $vec1->[$index1][1] * $vec2->[$index2][1]);
            ++$index1;
            ++$index2;
        }
        elsif ($vec1->[$index1][0] < $vec2->[$index2][0]) {
            $index1++;
        }
        else {
            $index2++;
        }
    }
    return $sim;
}

sub get_vector_from_text {
    my $line = shift;
    my $vector_ref = [];
    while ($line =~ /(\S+):(\S+)/g) {
        push @$vector_ref, [$1, $2];
    }
    return $vector_ref;
}
